#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
主题: 创建数据处理管道
问题: 你想以数据管道(类似Unix管道)的方式迭代处理数据。 比如，你有个大量的数据需要处理，但是不能将它们一次性放入内存中。
提示 : 
"""

import os
import fnmatch
import gzip
import bz2
import re

def recipe1():

    pynames = gen_find('access-log*', '../')
    files = gen_opener(pynames)
    lines = gen_concatenate(files)
    printlines = gen_grep('(?i)python', lines)

    for line in printlines:
        print(f'{line = }')


def recipe2():
    '''计算出传输的字节数并计算其总和'''
    pynames = gen_find('access-log*', '../')
    files = gen_opener(pynames)
    lines = gen_concatenate(files)
    printlines = gen_grep('(?i)python', lines)

    bytecolumn = (line.rsplit(None,1)[1] for line in printlines)
    byteit = (int(x) for x in bytecolumn if x != '-')
    print('Total', sum(byteit))

def gen_find(filepat, top):
    '''
    Find all filenames in a directory tree that match a shell wildcard pattern
    '''
    for path, _, filelist in os.walk(top):
        for name in fnmatch.filter(filelist, filepat):
            yield os.path.join(path, name)


def gen_opener(filenames):
    '''
    Open a sequence of filenames one at a time producing a file object.
    The file is closed immediately when proceeding to the next iteration.
    '''
    for filename in filenames:
        if filename.endswith('.gz'):
            f = gzip.open(filename, 'rt')
        elif filename.endswith('.bz2'):
            f = bz2.open(filename, 'rt')
        else:
            f = open(filename, 'rt')
        yield f
        f.close()


def gen_concatenate(iterators):
    '''
    Chain a sequence of iterators together into a single sequence.
    '''
    for it in iterators:
        yield from it


def gen_grep(pattern, lines):
    '''
    Look for a regex pattern in a sequence of lines
    '''
    pat = re.compile(pattern)
    for line in lines:
        if pat.search(line):
            yield line

def main():
    print('recipe1'.center(20, '*'))
    recipe1()
    print('recipe2'.center(20, '*'))
    recipe2()

if __name__ == '__main__':
    main()            

